Transform Weather Sensor Data from Waggle to an Xarray Dataset + Plot with ACT#

Imports#

import sage_data_client
from bokeh.models.formatters import DatetimeTickFormatter
import hvplot.pandas
import hvplot.xarray
import holoviews as hv
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import xarray as xr
import matplotlib.pyplot as plt
from metpy.plots import USCOUNTIES
import act
import numpy as np
import pandas as pd
import warnings
from bokeh.models import DatetimeTickFormatter

def apply_formatter(plot, element):
    plot.handles['xaxis'].formatter = DatetimeTickFormatter(hours='%m/%d/%Y \n %H:%M',
                                                            minutes='%m/%d/%Y \n %H:%M',
                                                            hourmin='%m/%d/%Y \n %H:%M',
                                                            days='%m/%d/%Y \n %H:%M',
                                                            months='%m/%d/%Y \n %H:%M')
xr.set_options(keep_attrs=True)
warnings.filterwarnings("ignore")
hv.extension("bokeh")

Query for the available Data#

wxt_df = sage_data_client.query(
    start="-3h",
    filter={
        "sensor": "vaisala-wxt536"
    }
)

Configure Helper Functions and Renaming Conventions#

The renaming is required due to . notations being problematic when working with both Pandas and Xarray data structures.

variable_rename_dict = {'wxt.env.humidity':'air_humidity',
                        'wxt.env.pressure':'air_pressure',
                        'wxt.env.temp':'air_temperature',
                        'wxt.heater.temp':'heater_temperature',
                        'wxt.heater.volt':'heater_voltage',
                        'wxt.rain.accumulation':'rain_accumulation',
                        'wxt.wind.direction':'wind_direction',
                        'wxt.wind.speed':'wind_speed',
                        'sys.gps.lat':'latitude',
                        'sys.gps.lon':'longitude',
                    }

def generate_data_array(df, variable, rename_variable_dict=variable_rename_dict):
    new_variable_name = rename_variable_dict[variable]
    df_variable= df.loc[df.name == variable]
    ds = df_variable.to_xarray().rename({'value':new_variable_name,
                                         'timestamp':'time',
                                         'meta.vsn':'node'})
    ds[new_variable_name].attrs['units'] = df_variable['meta.units'].values[0]
    ds['time'] = pd.to_datetime(ds.time)
    ds.attrs['datastream'] = ds.node.values[0]
    return ds[[new_variable_name]]

def generate_dataset(df, variables, rename_variable_dict=variable_rename_dict):
    reindexed = df.set_index(['meta.vsn', 'timestamp'])
    return xr.merge([generate_data_array(reindexed, variable) for variable in variables])

Transform the Data to Xarray#

wxt_variables = wxt_df.name.unique()
wxt_variables
array([], dtype=object)
wxt_ds = generate_dataset(wxt_df, wxt_variables).squeeze()
wxt_ds
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
/tmp/ipykernel_3917/1563727890.py in ?()
----> 1 wxt_ds = generate_dataset(wxt_df, wxt_variables).squeeze()
      2 wxt_ds

/tmp/ipykernel_3917/4226391414.py in ?(df, variables, rename_variable_dict)
     24 def generate_dataset(df, variables, rename_variable_dict=variable_rename_dict):
---> 25     reindexed = df.set_index(['meta.vsn', 'timestamp'])
     26     return xr.merge([generate_data_array(reindexed, variable) for variable in variables])

/usr/share/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/site-packages/pandas/util/_decorators.py in ?(*args, **kwargs)
    327                     msg.format(arguments=_format_argument_list(allow_args)),
    328                     FutureWarning,
    329                     stacklevel=find_stack_level(),
    330                 )
--> 331             return func(*args, **kwargs)

/usr/share/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/site-packages/pandas/core/frame.py in ?(self, keys, drop, append, inplace, verify_integrity)
   6008                     if not found:
   6009                         missing.append(col)
   6010 
   6011         if missing:
-> 6012             raise KeyError(f"None of {missing} are in the columns")
   6013 
   6014         if inplace:
   6015             frame = self

KeyError: "None of ['meta.vsn'] are in the columns"

Resample the data to minute requency#

minute_ds = wxt_ds.resample(time='1T').mean()

Visualize using hvplot#

meteogram_variables = ['air_temperature', 'air_humidity', 'wind_speed', 'wind_direction']
plots = []
for variable in meteogram_variables:
    plots.append(wxt_ds[variable].hvplot.line(label='10 Hz Data') * 
                 minute_ds[variable].hvplot.line(label='1 Minute Data'))
hv.Layout(plots).cols(2)

Plot Using the Atmospheric Data Community Toolkit (ACT)#

WindDisplay = act.plotting.WindRoseDisplay(minute_ds, figsize=(8, 10), subplot_shape=(1,))
WindDisplay.plot(
    'wind_direction', 'wind_speed', spd_bins=np.linspace(0, 25, 5), num_dirs=30, tick_interval=2, subplot_index=(0,)
)
<PolarAxes: title={'center': 'W057 on 20230414'}>
../../_images/7376c58a3d452189e8f3883abccbd95edf527aab3f20a12fbfffda2798b89cf1.png